# -*- coding:utf-8 -*-
import re
from selenium import webdriver
from lxml import etree
import time
from bs4 import BeautifulSoup
import requests
import random


def requestsurl(url, cookie, pagenum,columnId, ip=""):
    header = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Cache-Control': 'max-age=0',
        'Connection': 'keep-alive',
        'Cookie': 'td_cookie=2570783928; _gscu_1223948176=286460757ybvdr86; _gscbrs_1223948176=1; JSESSIONID=' + cookie + '; Hm_lvt_7030841eb7908f4f3779770e4a018fa8=1628646077,1628662825,1628735732,1628735734; Hm_lvt_04a971ecbbd7d7ba3d296a1439b6e74b=1628646075,1628662825,1628735732,1628735734; Hm_lpvt_04a971ecbbd7d7ba3d296a1439b6e74b=1628735741; Hm_lpvt_7030841eb7908f4f3779770e4a018fa8=1628735741; _gscs_1223948176=t28735735lx5spa20|pv:3',
        'Host': 'hnyzy.hicourt.gov.cn',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36',
    }

    params = {
        'currentPageNo': pagenum,
        'columnId': str(columnId),
        'pageSize': '10',
        '_': '1628735740{:0=3}'.format(int(pagenum) + 19),
    }
    if ip:
        proxies = ip
        result = requests.get(url=url, headers=header, params=params, proxies=proxies).text
    else:
        result = requests.get(url=url, params=params, headers=header).text

    return result


def chromesite(useragent="", ip="", headless=True, imge=False, linuxenv=False):
    chrome_options = webdriver.ChromeOptions()

    if useragent == "":
        chrome_options.add_argument(
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36")
    else:
        chrome_options.add_argument("user-agent:" + useragent)

    if headless == True:
        chrome_options.add_argument('--headless')

    if imge == False:
        prefs = {'profile.default_content_setting_values': {'images': 2}}
        chrome_options.add_experimental_option('prefs', prefs)

    if ip:
        chrome_options.add_argument('--proxy-server=' + ip)

    chrome_options.add_argument('--no-sandbox')

    if linuxenv == True:
        return webdriver.Chrome("./chromedriver", options=chrome_options)

    return webdriver.Chrome(options=chrome_options)


def randintnumber(numbers, system, usedlest, upstr=""):
    # upstr 是否使用大写字母 空为混用大小写bool
    # numbers 数字位数 int
    # system 进制 int
    # usedlest 已返回数字集合 set

    if system > 16:
        print("仅支持16进制以下进制随机数字！")
        return False
    while True:
        numbers = numbers
        if upstr == "True":
            strlist = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F']
        elif upstr == "False":
            strlist = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']
        elif upstr == "":
            strlist = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C',
                       'D', 'E', 'F']
            system += 6
        a = ""
        while numbers != 0:
            a += strlist[random.randint(0, int(system) - 1)]  # 随机数索引
            numbers -= 1
        if a in usedlest:
            continue
        else:
            usedlest.add(a)
            return a


def openIpihuan(pagenumber):
    driverIp = chromesite(linuxenv = False)
    driverIp.get("https://ip.ihuan.me/"+pagenumber)
    time.sleep(2)
    html = driverIp.page_source
    etr=etree.HTML(html)
    pageip=etr.xpath("//tr/td[1]/a/text() | //tr/td[2]/text()")
    pagenumber = etr.xpath("//ul[@class='pagination']/li[8]/a/@href")[0]
    iplist = []
    for i in range(0,len(pageip),2):
        ipdict = {}
        ipdict["http"] = "http://"+pageip[i]+":"+pageip[i+1]
        iplist.append(ipdict)

    driverIp.quit()
    return pagenumber,iplist

def trydaili(url,ip):
    try:
        headers = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'Accept-Encoding': 'gzip, deflate',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Connection': 'keep-alive',
            'Host': 'www.hicourt.gov.cn',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36',
        }
        r = requests.get(url=url,headers = headers,proxies=ip,timeout = 15)
        if r.status_code == 200:
            return ip
    except Exception as e:
        print(e)



def okip(pageip):
    iplist = []
    for i in pageip:
        if len(iplist) > 2:
            return iplist
        print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
        print(i)
        a = trydaili(openurl, i)
        if a != None:
            iplist.append(a)
            print(a, "ok")
    return iplist


timepot13 = int(time.time() * 1000)
timepot10 = int(time.time())

use32set = set()
cookie = "55F8E3713ED59D979BE04F05FAD8C42A"
# lastpage =


urllist = [
    #[
    #    "http://qzfy.hicourt.gov.cn/preview/channel?columnId=32837c25-e866-4773-ac1b-5943b2a17bad&&parentId=9ff4b7d7-8d5a-4c77-854c-4b34e05466e2&&siteId=1f191842-89d3-438d-80d6-e2400a275c51",
    #    "32837c25-e866-4773-ac1b-5943b2a17bad", 10,"/html/body/div[4]/div/div[3]//text()"],
    #[
    #    "http://wcfy.hicourt.gov.cn/preview/channel?columnId=abeb65c3-dff9-4633-8d29-50932f6512ea&&parentId=73ee198b-e5de-4554-b2db-d3fdf67009fd&&siteId=9da38576-72d0-46a3-b15e-6720c16af934",
    #    "abeb65c3-dff9-4633-8d29-50932f6512ea", 10,"/html/body/div[3]/div[2]/div[2]//text()"],
    #[
    #    "http://mlfy.hicourt.gov.cn/preview/channel?columnId=ea857fa2-9a6e-4bc5-9b44-f2d481816745&&parentId=716f323f-5258-4d27-9de6-6d390fc2bb04&&siteId=ecec453f-a3b7-41bd-8989-c6b5fd93ad82",
    #    "ea857fa2-9a6e-4bc5-9b44-f2d481816745", 10,"/html/body/div[4]/div/div[3]//text()"],
    #[
    #    "http://dzfy.hicourt.gov.cn/preview/channel?columnId=a017e8fe-3a9c-44f7-b3cc-1e8673c95f8f&&parentId=139c1da9-0466-49c5-ae49-3a8dea1320ef&&siteId=8daf1d42-6797-4d1c-8ec1-18c7fe42590d",
    #    "a017e8fe-3a9c-44f7-b3cc-1e8673c95f8f",10,"/html/body/div[3]/div[2]/div[2]/div/div//text()"],
    #[
    #    "http://lsfy.hicourt.gov.cn/preview/channel?columnId=354a71b2-93c7-4604-be80-dd86ad5b29b4&&parentId=cec856aa-27e2-44fc-982b-32cddcc84b1b&&siteId=17681f8a-e974-4543-8ad6-1240080712b2",
    #    "354a71b2-93c7-4604-be80-dd86ad5b29b4", 10,"/html/body/div[4]/div/div[3]//text()"],
    #[
    #    "http://wzsfy.hicourt.gov.cn/preview/channel?columnId=f26a6e4f-3ac1-4aa1-a4fb-65ab8ffc41ad&&parentId=1ee18175-6535-4f4f-8281-ebfb03e68146&&siteId=7aef3eaa-d649-44af-ac1a-726eb2f34c12",
    #    "f26a6e4f-3ac1-4aa1-a4fb-65ab8ffc41ad", 10,"/html/body/div[3]/div[2]/div[2]//text()"],
    #[
    #    "http://tcfy.hicourt.gov.cn/preview/channel?columnId=c660ecd1-4726-4f0d-810e-4b476a0fc1a7&&parentId=a999555d-4dcc-4332-a664-095fa0c8a840&&siteId=db6adfce-2502-4cea-82af-599b1e955288",
    #    "c660ecd1-4726-4f0d-810e-4b476a0fc1a7", 10,"/html/body/div[3]/div[2]/div/div[2]//text()"],
    #[
    #    "http://wnfy.hicourt.gov.cn/preview/channel?columnId=63328c85-45d0-4ad8-ac66-4548e5d3816c&&parentId=a63998bb-b5ae-4395-a32f-fa9ccca3561f&&siteId=0b075a78-5241-4146-8087-9511923de9ec",
    #    "63328c85-45d0-4ad8-ac66-4548e5d3816c", 10,"/html/body/div[4]/div/div[3]//text()"],
    #[
    #    "http://qhfy.hicourt.gov.cn/preview/channel?columnId=b30abdd1-5762-4aa0-9c7e-a2bd22a2b08b&&parentId=0f35997f-22c8-458f-8644-641d6f3ea7ac&&siteId=d012e5c2-aad2-49a0-998a-0a9cfe04530b",
    #    "b30abdd1-5762-4aa0-9c7e-a2bd22a2b08b", 10,"/html/body/div[3]/div[2]/div/div[2]//text()"],
    #[
    #    "http://dafy.hicourt.gov.cn/preview/channel?columnId=1628366d-69a0-4c4b-914d-5919fee07c72&&parentId=5d60fc75-829e-43ec-8702-271f33021fc7&&siteId=f5b8b1e0-0b8d-42a7-85e5-12e68ae4bdb0",
    #    "1628366d-69a0-4c4b-914d-5919fee07c72", 10,"/html/body/div[3]/div[2]/div[2]//text()"],
    #[
    #    "http://syzy.hicourt.gov.cn/preview/channel?columnId=905f2750-99eb-41cd-b75d-77fa7c926b8e&&parentId=71a48bad-910d-4bb6-9b34-6e320f2a3583&&siteId=bebfd487-19aa-4c65-acc0-6666b6d676ba",
    #    "905f2750-99eb-41cd-b75d-77fa7c926b8e", 20,"/html/body/div[3]/div/div[2]/div[2]//text()"],
    #[
    #    "http://bsfy.hicourt.gov.cn/preview/channel?columnId=9b07c25f-33f6-4691-90ed-fb06326df155&&parentId=fa828c4d-7ebf-4e01-b2c3-7f083c6160a9&&siteId=e98807e1-3467-4185-a854-d6158aa9e7da",
    #    "9b07c25f-33f6-4691-90ed-fb06326df155", 20,"/html/body/div[3]/div/div[2]/div[2]//text()"],
    #[
    #    "http://cjfy.hicourt.gov.cn/preview/channel?columnId=e0c16712-b512-499e-a194-16650b4f72b0&&parentId=4619691c-85b1-4164-990f-988c0baae8c6&&siteId=715d43d9-cfed-4e53-aa5a-f7e2a9d7227c",
    #    "e0c16712-b512-499e-a194-16650b4f72b0", 20,"/html/body/div[3]/div/div[2]/div[2]//text()"],
    #[
    #    "http://ldfy.hicourt.gov.cn/preview/channel?columnId=50ea407e-e4e4-4416-8f72-3931bd2a9caf&&parentId=e3c6c544-3e3c-46ca-927d-ba2ca8172a85&&siteId=0c511c1e-d2ca-420d-89b3-f0adefa2b976",
    #    "50ea407e-e4e4-4416-8f72-3931bd2a9caf", 20,"/html/body/div[3]/div/div[2]/div[2]//text()"],
    #[
    #    "http://lgfy.hicourt.gov.cn/preview/channel?columnId=1e1cf37c-6064-4168-8d52-195afaa2ec14&&parentId=06d5cdf5-c608-4692-ad13-1b1f1f1b72c1&&siteId=da56890c-9ecd-457f-b693-fc7cff7a92e3",
    #    "1e1cf37c-6064-4168-8d52-195afaa2ec14", 20,"/html/body/div[3]/div/div[2]/div[2]//text()"],
    [
        "http://btfy.hicourt.gov.cn/preview/channel?columnId=a1396cbe-abe7-496d-9704-5baa90bc8315&&parentId=87fd5273-e986-4241-8af8-91aae2d82b8c&&siteId=18ad6537-e2df-4a68-ada7-70cef1240a2c",
        "a1396cbe-abe7-496d-9704-5baa90bc8315", 20,"/html/body/div[3]/div/div[2]/div[2]//text()"],
    [
        "http://sycj.hicourt.gov.cn/preview/channel?columnId=c9e722fd-e2c3-4f4a-ab49-c522e1c949bc&&parentId=ecc5b99a-aef5-45d2-8ae2-fc21670949d1&&siteId=62bdaa82-b2ca-4386-840c-27f0b5a36c92",
        "c9e722fd-e2c3-4f4a-ab49-c522e1c949bc", 20,"/html/body/div[3]/div/div[2]/div[2]//text()"],
    [
        "http://qsfy.hicourt.gov.cn/preview/channel?columnId=6a1c7101-ff3a-4c27-bb14-5bbf4fd93bcc&&parentId=b36ce524-9e8c-45f4-b100-a164a1234a81&&siteId=0262906b-2b9c-455c-bf1f-44f1784ba330",
        "6a1c7101-ff3a-4c27-bb14-5bbf4fd93bcc", 30,"/html/body/div[3]/div[2]/div/div[3]//text()"],
    [
        "http://xyfy.hicourt.gov.cn/preview/channel?columnId=ecf3194d-a332-430c-b9b6-4d36adfbf45f&&parentId=dd7fd507-d464-48a4-becc-cb01aff0c901&&siteId=a82def60-9a0b-48e8-aba2-909978e3e199",
        "ecf3194d-a332-430c-b9b6-4d36adfbf45f", 30,"/html/body/div[3]/div[2]/div/div[3]//text()"],
    [
        "http://lhfy.hicourt.gov.cn/preview/channel?columnId=aecd2c85-b459-4dfb-b077-a5d9a40c30db&&parentId=abfad1ab-96a9-4245-8017-1346d8520575&&siteId=7c72c3db-b2b9-4bc6-8abe-741bdd584a6e",
        "aecd2c85-b459-4dfb-b077-a5d9a40c30db", 30,"/html/body/div[3]/div[2]/div/div[3]//text()"],
    [
        "http://dffy.hicourt.gov.cn/preview/channel?columnId=20b94483-dc98-4801-8d61-0bd09f078719&&parentId=213b927f-82a7-4860-8d3b-bd488ada187a&&siteId=e2a163de-c032-433a-80f0-c3444367b834",
        "20b94483-dc98-4801-8d61-0bd09f078719", 30,"/html/body/div[4]/div/div[2]//text"],
]
nowpage = 38

openurl = "http://www.hicourt.gov.cn/"

pagenumber = "?page=1"
openiplist = []
ip = ""



for url in urllist:
    while True:
        try:
            if openiplist == []:
                pageinfo = openIpihuan(pagenumber)
                pagenumber = pageinfo[0]
                pageip = pageinfo[1]
                openiplist = okip(pageip)

            driverspage = chromesite(linuxenv = False)
            driverspage.get(url[0])
            time.sleep(10)
            html = driverspage.page_source
            etr = etree.HTML(html)
            lastpage = etr.xpath("//li[@page-rel='lastpage']/@page-data")[0]
            driverspage.quit()
            host = url[0].split("/")[2]
            print(host)
            for i in range(nowpage,int(lastpage)):
                nowpage = i
                print(i,"/",lastpage, "page")
                print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
                hngy = requestsurl("http://"+host+"/preview/getArticleByColumn?currentPageNo=" + str(
                    i + 1) + "&columnId="+url[1]+"&pageSize="+str(url[2])+"&_=" + str(timepot13),
                                   cookie,
                                   str(i + 1),
                                   str(url[1]),
                                   )
                # print(hngy)
                idlist = re.findall("id\":\"(.+?)\",\"createDate", hngy)
                collist = re.findall("columnId\":\"(.+?)\",\"columnName", hngy)
                sitelist = re.findall("siteId\":\"(.+?)\",\"title", hngy)
                for id in range(len(idlist)):
                    print(id, "dot")
                    texturl = "http://"+host+"/preview/article?articleId=" + idlist[id] + "&&colArticleId=" + \
                              collist[
                                  id] + "&&siteId=" + sitelist[id]

                    print(texturl)
                    drivers = chromesite(linuxenv = False)
                    drivers.get(texturl)

                    time.sleep(20)

                    html = drivers.page_source
                    etr = etree.HTML(html)
                    text = etr.xpath(url[3])

                    with open("hnfy.txt", "a", encoding="utf-8") as w:
                        for t in text:
                            w.write(t.strip())
                        w.write("\n")
                    with open("hainanlog.txt", "a", encoding="utf-8") as w:
                        w.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
                        w.write("\t")
                        w.write(host)
                        w.write("\t")
                        w.write(str(i))
                        w.write("\t")
                        w.write(str(id))
                        w.write("\t")
                        w.write(str(len(text)))
                        w.write("\n")
                    drivers.quit()
            nowpage = 0
            break
        except Exception as e:
            print(e)
            print("time.sleep(100)")
            time.sleep(100)
            cookie = randintnumber(32, 16, use32set, "True")
            if len(openiplist) > 0:
                ip = openiplist.pop(0)
            with open ("hainanerror.txt","a",encoding="utf-8") as w:
                w.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
                w.write("\t")
                w.write(str(e))
                w.write("\t")
                w.write(url[0])
                w.write("\t")
                w.write(str(nowpage))
                w.write("\n")
            continue
